Example:
simple flat
nested objects
lists of primitives
sparse data
coingecko coins
complex mixed
simple_flat.json — Original size: 763 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 763tokens: 384chars_og/tok: 2.0encoded_chars/tok: 2.0
[
· {
··· "id": 1,
··· "name": "Alice",
··· "role": "Engineer",
··· "department": "Backend"
· },
· {
··· "id": 2,
··· "name": "Bob",
··· "role": "Designer",
··· "department": "Frontend"
· },
· {
··· "id": 3,
··· "name": "Carol",
··· "role": "Manager",
··· "department": "Product"
· },
· {
··· "id": 4,
··· "name": "David",
··· "role": "Engineer",
··· "department": "Infrastructure"
· },
... (truncated)
JSON (min)
chars: 522tokens: 152chars_og/tok: 5.0encoded_chars/tok: 3.4
[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 95chars_og/tok: 8.0encoded_chars/tok: 2.5
id,name,role,department
1,Alice,Engineer,Backend
2,Bob,Designer,Frontend
3,Carol,Manager,Product
4,David,Engineer,Infrastructure
5,Eva,Analyst,Data
6,Frank,Engineer,Backend
7,Grace,Designer,Mobile
8,Henry,Manager,Engineering
TSV
chars: 234tokens: 95chars_og/tok: 8.0encoded_chars/tok: 2.5
idnameroledepartment
1AliceEngineerBackend
2BobDesignerFrontend
3CarolManagerProduct
4DavidEngineerInfrastructure
5EvaAnalystData
6FrankEngineerBackend
7GraceDesignerMobile
8HenryManagerEngineering
YAML
chars: 489tokens: 163chars_og/tok: 4.7encoded_chars/tok: 3.0
- department: Backend
· id: 1
· name: Alice
· role: Engineer
- department: Frontend
· id: 2
· name: Bob
· role: Designer
- department: Product
· id: 3
· name: Carol
· role: Manager
- department: Infrastructure
· id: 4
· name: David
· role: Engineer
- department: Data
· id: 5
· name: Eva
· role: Analyst
- department: Backend
· id: 6
· name: Frank
· role: Engineer
- department: Mobile
... (truncated)
TOON
chars: 246tokens: 98chars_og/tok: 7.8encoded_chars/tok: 2.5
[8]{id,name,role,department}:
· 1,Alice,Engineer,Backend
· 2,Bob,Designer,Frontend
· 3,Carol,Manager,Product
· 4,David,Engineer,Infrastructure
· 5,Eva,Analyst,Data
· 6,Frank,Engineer,Backend
· 7,Grace,Designer,Mobile
· 8,Henry,Manager,Engineering
TSON
chars: 229tokens: 90chars_og/tok: 8.5encoded_chars/tok: 2.5
{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 74chars_og/tok: 10.3encoded_chars/tok: 3.4
id; name; role; department
1; Alice; Engineer; Backend
2; Bob; Designer; Frontend
3; Carol; Manager; Product
4; David; Engineer; Infrastructure
5; Eva; Analyst; Data
6; Frank; Engineer; Backend
7; Grace; Designer; Mobile
8; Henry; Manager; Engineering
minemizer (compact)
chars: 224tokens: 85chars_og/tok: 9.0encoded_chars/tok: 2.6
id;name;role;department
1;Alice;Engineer;Backend
2;Bob;Designer;Frontend
3;Carol;Manager;Product
4;David;Engineer;Infrastructure
5;Eva;Analyst;Data
6;Frank;Engineer;Backend
7;Grace;Designer;Mobile
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 741tokens: 407chars_og/tok: 1.8encoded_chars/tok: 1.8
[
· {
··· "id": 1,
··· "user": {
····· "name": "Alice",
····· "email": "alice@example.com"
··· },
··· "status": "active"
· },
· {
··· "id": 2,
··· "user": {
····· "name": "Bob",
····· "email": "bob@example.com"
··· },
··· "status": "inactive"
· },
· {
··· "id": 3,
··· "user": {
····· "name": "Carol",
····· "email": "carol@example.com"
··· },
··· "status": "active"
· },
... (truncated)
JSON (min)
chars: 470tokens: 143chars_og/tok: 5.2encoded_chars/tok: 3.3
[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 195chars_og/tok: 3.8encoded_chars/tok: 2.4
- id: 1
· status: active
· user:
··· email: alice@example.com
··· name: Alice
- id: 2
· status: inactive
· user:
··· email: bob@example.com
··· name: Bob
- id: 3
· status: active
· user:
··· email: carol@example.com
··· name: Carol
- id: 4
· status: pending
· user:
··· email: david@example.com
··· name: David
- id: 5
· status: active
· user:
··· email: eva@example.com
··· name: Eva
... (truncated)
TOON
chars: 527tokens: 252chars_og/tok: 2.9encoded_chars/tok: 2.1
[6]:
· - id: 1
··· user:
····· name: Alice
····· email: alice@example.com
··· status: active
· - id: 2
··· user:
····· name: Bob
····· email: bob@example.com
··· status: inactive
· - id: 3
··· user:
····· name: Carol
····· email: carol@example.com
··· status: active
· - id: 4
··· user:
····· name: David
····· email: david@example.com
··· status: pending
· - id: 5
··· user:
····· name: Eva
····· email: eva@example.com
... (truncated)
TSON
chars: 249tokens: 101chars_og/tok: 7.3encoded_chars/tok: 2.5
{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 90chars_og/tok: 8.2encoded_chars/tok: 2.9
id; user{ name; email}; status
1; { Alice; alice@example.com}; active
2; { Bob; bob@example.com}; inactive
3; { Carol; carol@example.com}; active
4; { David; david@example.com}; pending
5; { Eva; eva@example.com}; active
6; { Frank; frank@example.com}; active
minemizer (compact)
chars: 232tokens: 95chars_og/tok: 7.8encoded_chars/tok: 2.4
id;user{ name;email};status
1;{Alice;alice@example.com};active
2;{Bob;bob@example.com};inactive
3;{Carol;carol@example.com};active
4;{David;david@example.com};pending
5;{Eva;eva@example.com};active
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 610tokens: 382chars_og/tok: 1.6encoded_chars/tok: 1.6
[
· {
··· "id": 1,
··· "name": "Alice",
··· "skills": [
····· "python",
····· "go",
····· "rust"
··· ]
· },
· {
··· "id": 2,
··· "name": "Bob",
··· "skills": [
····· "javascript",
····· "typescript"
··· ]
· },
· {
··· "id": 3,
··· "name": "Carol",
··· "skills": [
····· "java",
····· "kotlin",
····· "scala",
... (truncated)
JSON (min)
chars: 330tokens: 115chars_og/tok: 5.3encoded_chars/tok: 2.9
[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 153chars_og/tok: 4.0encoded_chars/tok: 2.2
- id: 1
· name: Alice
· skills:
· - python
· - go
· - rust
- id: 2
· name: Bob
· skills:
· - javascript
· - typescript
- id: 3
· name: Carol
· skills:
· - java
· - kotlin
· - scala
· - groovy
- id: 4
· name: David
· skills:
· - c
· - cpp
- id: 5
· name: Eva
... (truncated)
TOON
chars: 339tokens: 161chars_og/tok: 3.8encoded_chars/tok: 2.1
[6]:
· - id: 1
··· name: Alice
··· skills[3]: python,go,rust
· - id: 2
··· name: Bob
··· skills[2]: javascript,typescript
· - id: 3
··· name: Carol
··· skills[4]: java,kotlin,scala,groovy
· - id: 4
··· name: David
··· skills[2]: c,cpp
· - id: 5
··· name: Eva
··· skills[3]: ruby,elixir,erlang
· - id: 6
··· name: Frank
··· skills[1]: swift
TSON
chars: 168tokens: 80chars_og/tok: 7.6encoded_chars/tok: 2.1
{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 81chars_og/tok: 7.5encoded_chars/tok: 2.4
id; name; skills[]
1; Alice; [ python; go; rust]
2; Bob; [ javascript; typescript]
3; Carol; [ java; kotlin; scala; groovy]
4; David; [ c; cpp]
5; Eva; [ ruby; elixir; erlang]
6; Frank; [ swift]
minemizer (compact)
chars: 165tokens: 83chars_og/tok: 7.3encoded_chars/tok: 2.0
id;name;skills[]
1;Alice;[python;go;rust]
2;Bob;[javascript;typescript]
3;Carol;[java;kotlin;scala;groovy]
4;David;[c;cpp]
5;Eva;[ruby;elixir;erlang]
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 589tokens: 318chars_og/tok: 1.9encoded_chars/tok: 1.9
[
· {
··· "id": 1,
··· "name": "Carol",
··· "role": "Manager"
· },
· {
··· "id": 2,
··· "name": "Dave",
··· "remote": true
· },
· {
··· "id": 3,
··· "name": "Eve",
··· "role": "Designer",
··· "team": "UX"
· },
· {
··· "id": 4,
··· "name": "Frank",
··· "department": "Engineering"
· },
· {
··· "id": 5,
··· "name": "Grace",
... (truncated)
JSON (min)
chars: 378tokens: 121chars_og/tok: 4.9encoded_chars/tok: 3.1
[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 135chars_og/tok: 4.4encoded_chars/tok: 2.6
- id: 1
· name: Carol
· role: Manager
- id: 2
· name: Dave
· remote: true
- id: 3
· name: Eve
· role: Designer
· team: UX
- department: Engineering
· id: 4
· name: Frank
- id: 5
· name: Grace
· remote: true
· role: Engineer
· team: Platform
- id: 6
· name: Henry
· role: Analyst
- id: 7
· name: Ivy
- department: Sales
· id: 8
... (truncated)
TOON
chars: 414tokens: 184chars_og/tok: 3.2encoded_chars/tok: 2.2
[8]:
· - id: 1
··· name: Carol
··· role: Manager
· - id: 2
··· name: Dave
··· remote: true
· - id: 3
··· name: Eve
··· role: Designer
··· team: UX
· - id: 4
··· name: Frank
··· department: Engineering
· - id: 5
··· name: Grace
··· role: Engineer
··· remote: true
··· team: Platform
· - id: 6
··· name: Henry
··· role: Analyst
· - id: 7
··· name: Ivy
· - id: 8
... (truncated)
TSON
chars: 300tokens: 136chars_og/tok: 4.3encoded_chars/tok: 2.2
[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 79chars_og/tok: 7.5encoded_chars/tok: 2.9
id; name; role
1; Carol; Manager
2; Dave; ; remote:True
3; Eve; Designer; team:UX
4; Frank; ; department:Engineering
5; Grace; Engineer; remote:True; team:Platform
6; Henry; Analyst
7; Ivy;·
8; Jack; ; department:Sales; remote:False
minemizer (compact)
chars: 207tokens: 84chars_og/tok: 7.0encoded_chars/tok: 2.5
id;name;role
1;Carol;Manager
2;Dave;;remote:True
3;Eve;Designer;team:UX
4;Frank;;department:Engineering
5;Grace;Engineer;remote:True;team:Platform
6;Henry;Analyst
7;Ivy;
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 1,611,780tokens: 862,469chars_og/tok: 1.9encoded_chars/tok: 1.9
[
· {
··· "id": "_",
··· "symbol": "gib",
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"
· },
· {
··· "id": "000-capital",
··· "symbol": "000",
··· "name": "000 Capital"
· },
· {
··· "id": "01111010011110000110001001110100-token",
··· "symbol": "01111010011110000110001001110100",
··· "name": "01111010011110000110001001110100"
· },
· {
··· "id": "01-token",
··· "symbol": "01",
··· "name": "01"
· },
· {
··· "id": "0chain",
··· "symbol": "zcn",
··· "name": "Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 400,135chars_og/tok: 4.0encoded_chars/tok: 2.9
[{"id":"_","symbol":"gib","name":"\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000 Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIX Protocol"},{"id":"0x","symbol":"zrx","name":"0x Protocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai: AI Smart Contract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0x Leverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonk by Virtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"Avalanche Bridged ZRX (Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsim by Virtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS (Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000x by Virtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10 figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1 Coin Can Change Your Life"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1 community can change your life"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1 dog can change your life"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1Hive Water"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCH yVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1Move Token"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1 narrative can change your life"},{"id":"1-one","symbol":"one","name":"1 (one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUS DAO"},{"id":"1-squirrel","symbol":"peanut","name":"OG Peanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004 PEPE"},{"id":"2025-token","symbol":"2025","name":"2025 TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077 CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23 Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3 years and forget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24K Gold PEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2G Carbon Coin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"Meow Meow Coin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACE ART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
CSV
chars: 603,635tokens: 282,641chars_og/tok: 5.7encoded_chars/tok: 2.1
id,symbol,name
_,gib,_·
000-capital,000,000 Capital
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100
01-token,01,01
0chain,zcn,Zus
0vix-protocol,vix,0VIX Protocol
0x,zrx,0x Protocol
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract
0x678-landwolf-1933,wolf,Landwolf
0xgasless-2,0xgas,0xGasless
0xgen,xgn,0xGen
0x-leverage,oxl,0x Leverage
0xlsd,0xlsd,0xLSD
0xmonk,monk,0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)
0xshadow,0xs,0xShadow
0xsim-by-virtuals,sage,0xsim by Virtuals
0xy,0xy,0xy
-10,loong,
1000bonk,1000bonk,1000BONK
1000btt,1000btt,1000BTT
1000cat,1000cat,1000CAT
1000chems,1000cheems,1000CHEMS
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 282,672chars_og/tok: 5.7encoded_chars/tok: 2.1
idsymbolname
_gib_·
000-capital000000 Capital
01111010011110000110001001110100-token0111101001111000011000100111010001111010011110000110001001110100
01-token0101
0chainzcnZus
0vix-protocolvix0VIX Protocol
0xzrx0x Protocol
0x0-ai-ai-smart-contract0x00x0.ai: AI Smart Contract
0x678-landwolf-1933wolfLandwolf
0xgasless-20xgas0xGasless
0xgenxgn0xGen
0x-leverageoxl0x Leverage
0xlsd0xlsd0xLSD
0xmonkmonk0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-ezrx.eAvalanche Bridged ZRX (Avalanche)
0xshadow0xs0xShadow
0xsim-by-virtualssage0xsim by Virtuals
0xy0xy0xy
-10loong
1000bonk1000bonk1000BONK
1000btt1000btt1000BTT
1000cat1000cat1000CAT
1000chems1000cheems1000CHEMS
1000mog1000mog1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 429,115chars_og/tok: 3.8encoded_chars/tok: 2.4
- id: _
· name:·_·
· symbol: gib
- id: 000-capital
· name: 000 Capital
· symbol: '000'
- id: 01111010011110000110001001110100-token
· name: '01111010011110000110001001110100'
· symbol: '01111010011110000110001001110100'
- id: 01-token
· name: '01'
· symbol: '01'
- id: 0chain
· name: Zus
· symbol: zcn
- id: 0vix-protocol
· name: 0VIX Protocol
· symbol: vix
- id: 0x
· name: 0x Protocol
· symbol: zrx
- id: 0x0-ai-ai-smart-contract
· name: '0x0.ai: AI Smart Contract'
· symbol: '0x0'
- id: 0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 279,372chars_og/tok: 5.8encoded_chars/tok: 2.2
[19332]{id,symbol,name}:
· _,gib,_·
· 000-capital,"000",000 Capital
· 01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"
· 01-token,"01","01"
· 0chain,zcn,Zus
· 0vix-protocol,vix,0VIX Protocol
· 0x,zrx,0x Protocol
· 0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"
· 0x678-landwolf-1933,wolf,Landwolf
· 0xgasless-2,0xgas,0xGasless
· 0xgen,xgn,0xGen
· 0x-leverage,oxl,0x Leverage
· 0xlsd,0xlsd,0xLSD
· 0xmonk,monk,0xMonk by Virtuals
· 0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)
· 0xshadow,0xs,0xShadow
· 0xsim-by-virtuals,sage,0xsim by Virtuals
· 0xy,0xy,0xy
· "-10",loong,
· 1000bonk,1000bonk,1000BONK
· 1000btt,1000btt,1000BTT
· 1000cat,1000cat,1000CAT
· 1000chems,1000cheems,1000CHEMS
· 1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 271,823chars_og/tok: 5.9encoded_chars/tok: 2.2
{@id,symbol,name#19332|_,gib,"_·"|000-capital,"000","000 Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIX Protocol"|0x,zrx,"0x Protocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0x Leverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonk by Virtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"Avalanche Bridged ZRX (Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsim by Virtuals"|0xy,0xy,0xy|"-10",loong,|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS (Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000x by Virtuals"|100-token,100,100|100xdarren,100x,100xDarren|10-figs,figs,"10 figs"|"-11",,|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1 Coin Can Change Your Life"|1-community-can-change-your-life,community,"1 community can change your life"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1 dog can change your life"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1Hive Water"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCH yVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1Move Token"|1-narrative-can-change-your-life,narrative,"1 narrative can change your life"|1-one,one,"1 (one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUS DAO"|1-squirrel,peanut,"OG Peanut"|1-token,"1","1"|2004-pepe,bog,"2004 PEPE"|2025-token,"2025","2025 TOKEN"|2077-code,"2077","2077 CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23 Turtles"|2-3-years-and-forget,"23","2-3 years and forget"|24k-gold-pepe,goldpepe,"24K Gold PEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2G Carbon Coin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"Meow Meow Coin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACE ART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47th POTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4 Next Unicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TB Coin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-Way Mirror Money"|"-5",,|500m-piece-of-paper,paper,"$500M piece of paper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5th Scape"|"-6","",""|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",,Voidify|717ai-by-virtuals,wire,"717ai by Virtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",,|8004-dog,dog8004,"8004 Dog"|8008-token,"8008","8008"|888coin,,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8 Ball"|8-bit-coin,coin,"8-Bit Coin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",,|9-5,9-5,9to5|99-bitcoins,99btc,"99 Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16z AI Dog"|a51-finance,a51,"A51 Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaa cat"|aaai_agent-by-virtuals,aaai,"AAAI_agent by Virtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADex Finance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"Aark Digital"|aarna-afi-802v2,"afi 802v2","aarna afi 802v2"|aarna-atv111,atv111,"aarna atv111"|aarna-atv111-arbitrum,atv111,"aarna atv111 (Arbitrum)"|aarna-atv111-sonic,atv111,"aarna atv111 (Sonic)"|aarna-atv-808,atv808,"aarna atv 808"|aarna-atv-usdc,atvusdc,"aarna atv USDC (Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarna atv USDC (Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"Aave AAVE"|aave-amm-bptbalweth,aammbptbalweth,"Aave AMM BptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"Aave AMM BptWBTCWETH"|aave-amm-dai,aammdai,"Aave AMM DAI"|aave-amm-uniaaveweth,aammuniaaveweth,"Aave AMM UniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"Aave AMM UniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"Aave AMM UniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"Aave AMM UniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"Aave AMM UniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"Aave AMM UniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"Aave AMM·
... (truncated)
minemizer
chars: 622,925tokens: 258,209chars_og/tok: 6.2encoded_chars/tok: 2.4
id; symbol; name
_; gib;·_·
000-capital; 000; 000 Capital
01111010011110000110001001110100-token; 01111010011110000110001001110100; 01111010011110000110001001110100
01-token; 01; 01
0chain; zcn; Zus
0vix-protocol; vix; 0VIX Protocol
0x; zrx; 0x Protocol
0x0-ai-ai-smart-contract; 0x0; 0x0.ai: AI Smart Contract
0x678-landwolf-1933; wolf; Landwolf
0xgasless-2; 0xgas; 0xGasless
0xgen; xgn; 0xGen
0x-leverage; oxl; 0x Leverage
0xlsd; 0xlsd; 0xLSD
0xmonk; monk; 0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)
0xshadow; 0xs; 0xShadow
0xsim-by-virtuals; sage; 0xsim by Virtuals
0xy; 0xy; 0xy
-10; loong;·
1000bonk; 1000bonk; 1000BONK
1000btt; 1000btt; 1000BTT
1000cat; 1000cat; 1000CAT
1000chems; 1000cheems; 1000CHEMS
1000mog; 1000mog; 1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 263,320chars_og/tok: 6.1encoded_chars/tok: 2.2
id;symbol;name
_;gib;_·
000-capital;000;000 Capital
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100
01-token;01;01
0chain;zcn;Zus
0vix-protocol;vix;0VIX Protocol
0x;zrx;0x Protocol
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract
0x678-landwolf-1933;wolf;Landwolf
0xgasless-2;0xgas;0xGasless
0xgen;xgn;0xGen
0x-leverage;oxl;0x Leverage
0xlsd;0xlsd;0xLSD
0xmonk;monk;0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)
0xshadow;0xs;0xShadow
0xsim-by-virtuals;sage;0xsim by Virtuals
0xy;0xy;0xy
-10;loong;
1000bonk;1000bonk;1000BONK
1000btt;1000btt;1000BTT
1000cat;1000cat;1000CAT
1000chems;1000cheems;1000CHEMS
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 1,320tokens: 768chars_og/tok: 1.7encoded_chars/tok: 1.7
[
· {
··· "id": 1,
··· "profile": {
····· "name": "Grace",
····· "location": {
······· "city": "NYC",
······· "country": "USA"
····· }
··· },
··· "tags": [
····· "admin",
····· "verified"
··· ],
··· "metadata": {
····· "created": "2024-01-15"
··· }
· },
· {
··· "id": 2,
··· "profile": {
····· "name": "Henry",
····· "location": {
······· "city": "London",
······· "country": "UK"
... (truncated)
JSON (min)
chars: 760tokens: 224chars_og/tok: 5.9encoded_chars/tok: 3.4
[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 374chars_og/tok: 3.5encoded_chars/tok: 2.2
- id: 1
· metadata:
··· created: '2024-01-15'
· profile:
··· location:
····· city: NYC
····· country: USA
··· name: Grace
· tags:
· - admin
· - verified
- id: 2
· metadata:
··· created: '2024-02-20'
··· updated: '2024-03-10'
· profile:
··· location:
····· city: London
····· country: UK
··· name: Henry
· tags:
· - user
- id: 3
· metadata:
··· created: '2024-01-05'
... (truncated)
TOON
chars: 881tokens: 434chars_og/tok: 3.0encoded_chars/tok: 2.0
[5]:
· - id: 1
··· profile:
····· name: Grace
····· location:
······· city: NYC
······· country: USA
··· tags[2]: admin,verified
··· metadata:
····· created: 2024-01-15
· - id: 2
··· profile:
····· name: Henry
····· location:
······· city: London
······· country: UK
··· tags[1]: user
··· metadata:
····· created: 2024-02-20
····· updated: 2024-03-10
· - id: 3
··· profile:
····· name: Ivy
····· location:
······· city: Tokyo
... (truncated)
TSON
chars: 453tokens: 207chars_og/tok: 6.4encoded_chars/tok: 2.2
{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 159chars_og/tok: 8.3encoded_chars/tok: 2.6
id; profile{ name; location{ city; country}}; tags[]; metadata{ created; ...}
1; { Grace; { NYC; USA}}; [ admin; verified]; { 2024-01-15}
2; { Henry; { London; UK}}; [ user]; { 2024-02-20; updated:2024-03-10}
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; { 2024-01-05}
4; { Jack; { Sydney; Australia}}; [ user; new]; { 2024-04-01}
5; { Kate; { Berlin; Germany}}; [ admin]; { 2023-12-01; updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 173chars_og/tok: 7.6encoded_chars/tok: 2.1
id;profile{ name;location{ city;country}};tags[];metadata{ created;...}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
simple_flat.json — Original size: 763 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 763tokens: 334chars_og/tok: 2.3encoded_chars/tok: 2.3
<s>[
{
··"id":1,
··"name":"Alice",
··"role":"Engineer",
··"department":"Backend"
},
{
··"id":2,
··"name":"Bob",
··"role":"Designer",
··"department":"Frontend"
},
{
··"id":3,
··"name":"Carol",
··"role":"Manager",
··"department":"Product"
},
{
··"id":4,
··"name":"David",
··"role":"Engineer",
··"department":"Infrastructure"
},
... (truncated)
JSON (min)
chars: 522tokens: 165chars_og/tok: 4.6encoded_chars/tok: 3.2
<s>[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 101chars_og/tok: 7.6encoded_chars/tok: 2.3
<s>id,name,role,department
1,Alice,Engineer,Backend
2,Bob,Designer,Frontend
3,Carol,Manager,Product
4,David,Engineer,Infrastructure
5,Eva,Analyst,Data
6,Frank,Engineer,Backend
7,Grace,Designer,Mobile
8,Henry,Manager,Engineering
TSV
chars: 234tokens: 101chars_og/tok: 7.6encoded_chars/tok: 2.3
<s>idnameroledepartment
1AliceEngineerBackend
2BobDesignerFrontend
3CarolManagerProduct
4DavidEngineerInfrastructure
5EvaAnalystData
6FrankEngineerBackend
7GraceDesignerMobile
8HenryManagerEngineering
YAML
chars: 489tokens: 180chars_og/tok: 4.2encoded_chars/tok: 2.7
<s>-department:Backend
id:1
name:Alice
role:Engineer
-department:Frontend
id:2
name:Bob
role:Designer
-department:Product
id:3
name:Carol
role:Manager
-department:Infrastructure
id:4
name:David
role:Engineer
-department:Data
id:5
name:Eva
role:Analyst
-department:Backend
id:6
name:Frank
role:Engineer
-department:Mobile
... (truncated)
TOON
chars: 246tokens: 103chars_og/tok: 7.4encoded_chars/tok: 2.4
<s>[8]{id,name,role,department}:
·1,Alice,Engineer,Backend
·2,Bob,Designer,Frontend
·3,Carol,Manager,Product
·4,David,Engineer,Infrastructure
·5,Eva,Analyst,Data
·6,Frank,Engineer,Backend
·7,Grace,Designer,Mobile
·8,Henry,Manager,Engineering
TSON
chars: 229tokens: 95chars_og/tok: 8.0encoded_chars/tok: 2.4
<s>{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 83chars_og/tok: 9.2encoded_chars/tok: 3.0
<s>id;name;role;department
1;Alice;Engineer;Backend
2;Bob;Designer;Frontend
3;Carol;Manager;Product
4;David;Engineer;Infrastructure
5;Eva;Analyst;Data
6;Frank;Engineer;Backend
7;Grace;Designer;Mobile
8;Henry;Manager;Engineering
minemizer (compact)
chars: 224tokens: 91chars_og/tok: 8.4encoded_chars/tok: 2.5
<s>id;name;role;department
1;Alice;Engineer;Backend
2;Bob;Designer;Frontend
3;Carol;Manager;Product
4;David;Engineer;Infrastructure
5;Eva;Analyst;Data
6;Frank;Engineer;Backend
7;Grace;Designer;Mobile
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 741tokens: 322chars_og/tok: 2.3encoded_chars/tok: 2.3
<s>[
{
··"id":1,
··"user":{
····"name":"Alice",
····"email":"alice@example.com"
··},
··"status":"active"
},
{
··"id":2,
··"user":{
····"name":"Bob",
····"email":"bob@example.com"
··},
··"status":"inactive"
},
{
··"id":3,
··"user":{
····"name":"Carol",
····"email":"carol@example.com"
··},
··"status":"active"
},
... (truncated)
JSON (min)
chars: 470tokens: 159chars_og/tok: 4.7encoded_chars/tok: 3.0
<s>[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 182chars_og/tok: 4.1encoded_chars/tok: 2.5
<s>-id:1
status:active
user:
··email:alice@example.com
··name:Alice
-id:2
status:inactive
user:
··email:bob@example.com
··name:Bob
-id:3
status:active
user:
··email:carol@example.com
··name:Carol
-id:4
status:pending
user:
··email:david@example.com
··name:David
-id:5
status:active
user:
··email:eva@example.com
··name:Eva
... (truncated)
TOON
chars: 527tokens: 191chars_og/tok: 3.9encoded_chars/tok: 2.8
<s>[6]:
-id:1
··user:
····name:Alice
····email:alice@example.com
··status:active
-id:2
··user:
····name:Bob
····email:bob@example.com
··status:inactive
-id:3
··user:
····name:Carol
····email:carol@example.com
··status:active
-id:4
··user:
····name:David
····email:david@example.com
··status:pending
-id:5
··user:
····name:Eva
····email:eva@example.com
... (truncated)
TSON
chars: 249tokens: 104chars_og/tok: 7.1encoded_chars/tok: 2.4
<s>{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 95chars_og/tok: 7.8encoded_chars/tok: 2.7
<s>id;user{name;email};status
1;{Alice;alice@example.com};active
2;{Bob;bob@example.com};inactive
3;{Carol;carol@example.com};active
4;{David;david@example.com};pending
5;{Eva;eva@example.com};active
6;{Frank;frank@example.com};active
minemizer (compact)
chars: 232tokens: 100chars_og/tok: 7.4encoded_chars/tok: 2.3
<s>id;user{name;email};status
1;{Alice;alice@example.com};active
2;{Bob;bob@example.com};inactive
3;{Carol;carol@example.com};active
4;{David;david@example.com};pending
5;{Eva;eva@example.com};active
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 610tokens: 280chars_og/tok: 2.2encoded_chars/tok: 2.2
<s>[
{
··"id":1,
··"name":"Alice",
··"skills":[
····"python",
····"go",
····"rust"
··]
},
{
··"id":2,
··"name":"Bob",
··"skills":[
····"javascript",
····"typescript"
··]
},
{
··"id":3,
··"name":"Carol",
··"skills":[
····"java",
····"kotlin",
····"scala",
... (truncated)
JSON (min)
chars: 330tokens: 125chars_og/tok: 4.9encoded_chars/tok: 2.6
<s>[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 157chars_og/tok: 3.9encoded_chars/tok: 2.2
<s>-id:1
name:Alice
skills:
-python
-go
-rust
-id:2
name:Bob
skills:
-javascript
-typescript
-id:3
name:Carol
skills:
-java
-kotlin
-scala
-groovy
-id:4
name:David
skills:
-c
-cpp
-id:5
name:Eva
... (truncated)
TOON
chars: 339tokens: 141chars_og/tok: 4.3encoded_chars/tok: 2.4
<s>[6]:
-id:1
··name:Alice
··skills[3]:python,go,rust
-id:2
··name:Bob
··skills[2]:javascript,typescript
-id:3
··name:Carol
··skills[4]:java,kotlin,scala,groovy
-id:4
··name:David
··skills[2]:c,cpp
-id:5
··name:Eva
··skills[3]:ruby,elixir,erlang
-id:6
··name:Frank
··skills[1]:swift
TSON
chars: 168tokens: 79chars_og/tok: 7.7encoded_chars/tok: 2.1
<s>{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 79chars_og/tok: 7.7encoded_chars/tok: 2.5
<s>id;name;skills[]
1;Alice;[python;go;rust]
2;Bob;[javascript;typescript]
3;Carol;[java;kotlin;scala;groovy]
4;David;[c;cpp]
5;Eva;[ruby;elixir;erlang]
6;Frank;[swift]
minemizer (compact)
chars: 165tokens: 83chars_og/tok: 7.3encoded_chars/tok: 2.0
<s>id;name;skills[]
1;Alice;[python;go;rust]
2;Bob;[javascript;typescript]
3;Carol;[java;kotlin;scala;groovy]
4;David;[c;cpp]
5;Eva;[ruby;elixir;erlang]
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 589tokens: 278chars_og/tok: 2.1encoded_chars/tok: 2.1
<s>[
{
··"id":1,
··"name":"Carol",
··"role":"Manager"
},
{
··"id":2,
··"name":"Dave",
··"remote":true
},
{
··"id":3,
··"name":"Eve",
··"role":"Designer",
··"team":"UX"
},
{
··"id":4,
··"name":"Frank",
··"department":"Engineering"
},
{
··"id":5,
··"name":"Grace",
... (truncated)
JSON (min)
chars: 378tokens: 133chars_og/tok: 4.4encoded_chars/tok: 2.8
<s>[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 150chars_og/tok: 3.9encoded_chars/tok: 2.4
<s>-id:1
name:Carol
role:Manager
-id:2
name:Dave
remote:true
-id:3
name:Eve
role:Designer
team:UX
-department:Engineering
id:4
name:Frank
-id:5
name:Grace
remote:true
role:Engineer
team:Platform
-id:6
name:Henry
role:Analyst
-id:7
name:Ivy
-department:Sales
id:8
... (truncated)
TOON
chars: 414tokens: 161chars_og/tok: 3.7encoded_chars/tok: 2.6
<s>[8]:
-id:1
··name:Carol
··role:Manager
-id:2
··name:Dave
··remote:true
-id:3
··name:Eve
··role:Designer
··team:UX
-id:4
··name:Frank
··department:Engineering
-id:5
··name:Grace
··role:Engineer
··remote:true
··team:Platform
-id:6
··name:Henry
··role:Analyst
-id:7
··name:Ivy
-id:8
... (truncated)
TSON
chars: 300tokens: 133chars_og/tok: 4.4encoded_chars/tok: 2.3
<s>[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 87chars_og/tok: 6.8encoded_chars/tok: 2.7
<s>id;name;role
1;Carol;Manager
2;Dave;;remote:True
3;Eve;Designer;team:UX
4;Frank;;department:Engineering
5;Grace;Engineer;remote:True;team:Platform
6;Henry;Analyst
7;Ivy;
8;Jack;;department:Sales;remote:False
minemizer (compact)
chars: 207tokens: 90chars_og/tok: 6.5encoded_chars/tok: 2.3
<s>id;name;role
1;Carol;Manager
2;Dave;;remote:True
3;Eve;Designer;team:UX
4;Frank;;department:Engineering
5;Grace;Engineer;remote:True;team:Platform
6;Henry;Analyst
7;Ivy;
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 1,611,780tokens: 749,813chars_og/tok: 2.1encoded_chars/tok: 2.1
<s>[
{
··"id":"_",
··"symbol":"gib",
··"name":"\u0f3c\u3064\u25d5_\u25d5\u0f3d\u3064"
},
{
··"id":"000-capital",
··"symbol":"000",
··"name":"000Capital"
},
{
··"id":"01111010011110000110001001110100-token",
··"symbol":"01111010011110000110001001110100",
··"name":"01111010011110000110001001110100"
},
{
··"id":"01-token",
··"symbol":"01",
··"name":"01"
},
{
··"id":"0chain",
··"symbol":"zcn",
··"name":"Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 422,810chars_og/tok: 3.8encoded_chars/tok: 2.7
<s>[{"id":"_","symbol":"gib","name":"\u0f3c\u3064\u25d5_\u25d5\u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIXProtocol"},{"id":"0x","symbol":"zrx","name":"0xProtocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai:AISmartContract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0xLeverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonkbyVirtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"AvalancheBridgedZRX(Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsimbyVirtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS(Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000xbyVirtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1CoinCanChangeYourLife"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1communitycanchangeyourlife"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1dogcanchangeyourlife"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1HiveWater"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCHyVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1MoveToken"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1narrativecanchangeyourlife"},{"id":"1-one","symbol":"one","name":"1(one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUSDAO"},{"id":"1-squirrel","symbol":"peanut","name":"OGPeanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004PEPE"},{"id":"2025-token","symbol":"2025","name":"2025TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3yearsandforget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24KGoldPEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2GCarbonCoin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"MeowMeowCoin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACEART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
CSV
chars: 603,635tokens: 303,115chars_og/tok: 5.3encoded_chars/tok: 2.0
<s>id,symbol,name
_,gib,_
000-capital,000,000Capital
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100
01-token,01,01
0chain,zcn,Zus
0vix-protocol,vix,0VIXProtocol
0x,zrx,0xProtocol
0x0-ai-ai-smart-contract,0x0,0x0.ai:AISmartContract
0x678-landwolf-1933,wolf,Landwolf
0xgasless-2,0xgas,0xGasless
0xgen,xgn,0xGen
0x-leverage,oxl,0xLeverage
0xlsd,0xlsd,0xLSD
0xmonk,monk,0xMonkbyVirtuals
0x-protocol-avalanche-bridged-zrx-e,zrx.e,AvalancheBridgedZRX(Avalanche)
0xshadow,0xs,0xShadow
0xsim-by-virtuals,sage,0xsimbyVirtuals
0xy,0xy,0xy
-10,loong,
1000bonk,1000bonk,1000BONK
1000btt,1000btt,1000BTT
1000cat,1000cat,1000CAT
1000chems,1000cheems,1000CHEMS
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 303,547chars_og/tok: 5.3encoded_chars/tok: 2.0
<s>idsymbolname
_gib_
000-capital000000Capital
01111010011110000110001001110100-token0111101001111000011000100111010001111010011110000110001001110100
01-token0101
0chainzcnZus
0vix-protocolvix0VIXProtocol
0xzrx0xProtocol
0x0-ai-ai-smart-contract0x00x0.ai:AISmartContract
0x678-landwolf-1933wolfLandwolf
0xgasless-20xgas0xGasless
0xgenxgn0xGen
0x-leverageoxl0xLeverage
0xlsd0xlsd0xLSD
0xmonkmonk0xMonkbyVirtuals
0x-protocol-avalanche-bridged-zrx-ezrx.eAvalancheBridgedZRX(Avalanche)
0xshadow0xs0xShadow
0xsim-by-virtualssage0xsimbyVirtuals
0xy0xy0xy
-10loong
1000bonk1000bonk1000BONK
1000btt1000btt1000BTT
1000cat1000cat1000CAT
1000chems1000cheems1000CHEMS
1000mog1000mog1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 454,247chars_og/tok: 3.5encoded_chars/tok: 2.3
<s>-id:_
name:_
symbol:gib
-id:000-capital
name:000Capital
symbol:'000'
-id:01111010011110000110001001110100-token
name:'01111010011110000110001001110100'
symbol:'01111010011110000110001001110100'
-id:01-token
name:'01'
symbol:'01'
-id:0chain
name:Zus
symbol:zcn
-id:0vix-protocol
name:0VIXProtocol
symbol:vix
-id:0x
name:0xProtocol
symbol:zrx
-id:0x0-ai-ai-smart-contract
name:'0x0.ai:AISmartContract'
symbol:'0x0'
-id:0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 301,477chars_og/tok: 5.3encoded_chars/tok: 2.1
<s>[19332]{id,symbol,name}:
_,gib,_
·000-capital,"000",000Capital
·01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"
·01-token,"01","01"
·0chain,zcn,Zus
·0vix-protocol,vix,0VIXProtocol
·0x,zrx,0xProtocol
·0x0-ai-ai-smart-contract,0x0,"0x0.ai:AISmartContract"
·0x678-landwolf-1933,wolf,Landwolf
·0xgasless-2,0xgas,0xGasless
·0xgen,xgn,0xGen
·0x-leverage,oxl,0xLeverage
·0xlsd,0xlsd,0xLSD
·0xmonk,monk,0xMonkbyVirtuals
·0x-protocol-avalanche-bridged-zrx-e,zrx.e,AvalancheBridgedZRX(Avalanche)
·0xshadow,0xs,0xShadow
·0xsim-by-virtuals,sage,0xsimbyVirtuals
·0xy,0xy,0xy
"-10",loong,
·1000bonk,1000bonk,1000BONK
·1000btt,1000btt,1000BTT
·1000cat,1000cat,1000CAT
·1000chems,1000cheems,1000CHEMS
·1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 293,257chars_og/tok: 5.5encoded_chars/tok: 2.1
<s>{@id,symbol,name#19332|_,gib,"_"|000-capital,"000","000Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIXProtocol"|0x,zrx,"0xProtocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai:AISmartContract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0xLeverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonkbyVirtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"AvalancheBridgedZRX(Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsimbyVirtuals"|0xy,0xy,0xy|"-10",loong,|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS(Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000xbyVirtuals"|100-token,100¥,100¥|100xdarren,100x,100xDarren|10-figs,figs,"10figs"|"-11",,|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1CoinCanChangeYourLife"|1-community-can-change-your-life,community,"1communitycanchangeyourlife"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1dogcanchangeyourlife"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1HiveWater"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCHyVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1MoveToken"|1-narrative-can-change-your-life,narrative,"1narrativecanchangeyourlife"|1-one,one,"1(one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUSDAO"|1-squirrel,peanut,"OGPeanut"|1-token,"1","1"|2004-pepe,bog,"2004PEPE"|2025-token,"2025","2025TOKEN"|2077-code,"2077","2077CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23Turtles"|2-3-years-and-forget,"23","2-3yearsandforget"|24k-gold-pepe,goldpepe,"24KGoldPEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2GCarbonCoin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"MeowMeowCoin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACEART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47thPOTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4NextUnicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TBCoin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-WayMirrorMoney"|"-5",,|500m-piece-of-paper,paper,"$500Mpieceofpaper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5thScape"|"-6"," "," "|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",,Voidify|717ai-by-virtuals,wire,"717aibyVirtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",,|8004-dog,dog8004,"8004Dog"|8008-token,"8008","8008"|888coin,,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8Ball"|8-bit-coin,coin,"8-BitCoin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",,|9-5,9-5,9to5|99-bitcoins,99btc,"99Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16zAIDog"|a51-finance,a51,"A51Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaacat"|aaai_agent-by-virtuals,aaai,"AAAI_agentbyVirtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADexFinance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"AarkDigital"|aarna-afi-802v2,"afi802v2","aarnaafi802v2"|aarna-atv111,atv111,"aarnaatv111"|aarna-atv111-arbitrum,atv111,"aarnaatv111(Arbitrum)"|aarna-atv111-sonic,atv111,"aarnaatv111(Sonic)"|aarna-atv-808,atv808,"aarnaatv808"|aarna-atv-usdc,atvusdc,"aarnaatvUSDC(Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarnaatvUSDC(Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"AaveAAVE"|aave-amm-bptbalweth,aammbptbalweth,"AaveAMMBptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"AaveAMMBptWBTCWETH"|aave-amm-dai,aammdai,"AaveAMMDAI"|aave-amm-uniaaveweth,aammuniaaveweth,"AaveAMMUniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"AaveAMMUniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"AaveAMMUniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"AaveAMMUniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"AaveAMMUniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"AaveAMMUniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"AaveAMM
... (truncated)
minemizer
chars: 622,925tokens: 282,585chars_og/tok: 5.7encoded_chars/tok: 2.2
<s>id;symbol;name
_;gib;_
000-capital;000;000Capital
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100
01-token;01;01
0chain;zcn;Zus
0vix-protocol;vix;0VIXProtocol
0x;zrx;0xProtocol
0x0-ai-ai-smart-contract;0x0;0x0.ai:AISmartContract
0x678-landwolf-1933;wolf;Landwolf
0xgasless-2;0xgas;0xGasless
0xgen;xgn;0xGen
0x-leverage;oxl;0xLeverage
0xlsd;0xlsd;0xLSD
0xmonk;monk;0xMonkbyVirtuals
0x-protocol-avalanche-bridged-zrx-e;zrx.e;AvalancheBridgedZRX(Avalanche)
0xshadow;0xs;0xShadow
0xsim-by-virtuals;sage;0xsimbyVirtuals
0xy;0xy;0xy
-10;loong;
1000bonk;1000bonk;1000BONK
1000btt;1000btt;1000BTT
1000cat;1000cat;1000CAT
1000chems;1000cheems;1000CHEMS
1000mog;1000mog;1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 285,149chars_og/tok: 5.7encoded_chars/tok: 2.0
<s>id;symbol;name
_;gib;_
000-capital;000;000Capital
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100
01-token;01;01
0chain;zcn;Zus
0vix-protocol;vix;0VIXProtocol
0x;zrx;0xProtocol
0x0-ai-ai-smart-contract;0x0;0x0.ai:AISmartContract
0x678-landwolf-1933;wolf;Landwolf
0xgasless-2;0xgas;0xGasless
0xgen;xgn;0xGen
0x-leverage;oxl;0xLeverage
0xlsd;0xlsd;0xLSD
0xmonk;monk;0xMonkbyVirtuals
0x-protocol-avalanche-bridged-zrx-e;zrx.e;AvalancheBridgedZRX(Avalanche)
0xshadow;0xs;0xShadow
0xsim-by-virtuals;sage;0xsimbyVirtuals
0xy;0xy;0xy
-10;loong;
1000bonk;1000bonk;1000BONK
1000btt;1000btt;1000BTT
1000cat;1000cat;1000CAT
1000chems;1000cheems;1000CHEMS
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 1,320tokens: 560chars_og/tok: 2.4encoded_chars/tok: 2.4
<s>[
{
··"id":1,
··"profile":{
····"name":"Grace",
····"location":{
······"city":"NYC",
······"country":"USA"
····}
··},
··"tags":[
····"admin",
····"verified"
··],
··"metadata":{
····"created":"2024-01-15"
··}
},
{
··"id":2,
··"profile":{
····"name":"Henry",
····"location":{
······"city":"London",
······"country":"UK"
... (truncated)
JSON (min)
chars: 760tokens: 284chars_og/tok: 4.6encoded_chars/tok: 2.7
<s>[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 338chars_og/tok: 3.9encoded_chars/tok: 2.4
<s>-id:1
metadata:
··created:'2024-01-15'
profile:
··location:
····city:NYC
····country:USA
··name:Grace
tags:
-admin
-verified
-id:2
metadata:
··created:'2024-02-20'
··updated:'2024-03-10'
profile:
··location:
····city:London
····country:UK
··name:Henry
tags:
-user
-id:3
metadata:
··created:'2024-01-05'
... (truncated)
TOON
chars: 881tokens: 329chars_og/tok: 4.0encoded_chars/tok: 2.7
<s>[5]:
-id:1
··profile:
····name:Grace
····location:
······city:NYC
······country:USA
··tags[2]:admin,verified
··metadata:
····created:2024-01-15
-id:2
··profile:
····name:Henry
····location:
······city:London
······country:UK
··tags[1]:user
··metadata:
····created:2024-02-20
····updated:2024-03-10
-id:3
··profile:
····name:Ivy
····location:
······city:Tokyo
... (truncated)
TSON
chars: 453tokens: 237chars_og/tok: 5.6encoded_chars/tok: 1.9
<s>{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 201chars_og/tok: 6.6encoded_chars/tok: 2.1
<s>id;profile{name;location{city;country}};tags[];metadata{created;...}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 214chars_og/tok: 6.2encoded_chars/tok: 1.7
<s>id;profile{name;location{city;country}};tags[];metadata{created;...}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
simple_flat.json — Original size: 763 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 763tokens: 264chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
JSON (min)
chars: 522tokens: 137chars_og/tok: 5.6encoded_chars/tok: 3.8
[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id,name,role,department
1,Alice,Engineer,Backend
2,Bob,Designer,Frontend
3,Carol,Manager,Product
4,David,Engineer,Infrastructure
5,Eva,Analyst,Data
6,Frank,Engineer,Backend
7,Grace,Designer,Mobile
8,Henry,Manager,Engineering
TSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id nameroledepartment
1 AliceEngineerBackend
2 BobDesigner Frontend
3 CarolManagerProduct
4 DavidEngineerInfrastructure
5 EvaAnalyst Data
6 FrankEngineerBackend
7GraceDesigner Mobile
8 HenryManagerEngineering
YAML
chars: 489tokens: 169chars_og/tok: 4.5encoded_chars/tok: 2.9
- department: Backend
· id:·1
· name: Alice
· role: Engineer
- department: Frontend
· id:·2
· name: Bob
· role: Designer
- department: Product
· id:·3
· name: Carol
· role: Manager
- department: Infrastructure
· id:·4
· name: David
· role: Engineer
- department: Data
· id:·5
· name: Eva
· role: Analyst
- department: Backend
· id:·6
· name: Frank
· role: Engineer
- department: Mobile
... (truncated)
TOON
chars: 246tokens: 96chars_og/tok: 7.9encoded_chars/tok: 2.6
[8]{id,name,role,department}:↵
··1,Alice,Engineer,Backend
··2,Bob,Designer,Frontend
··3,Carol,Manager,Product
··4,David,Engineer,Infrastructure
··5,Eva,Analyst,Data
··6,Frank,Engineer,Backend
··7,Grace,Designer,Mobile
··8,Henry,Manager,Engineering
TSON
chars: 229tokens: 80chars_og/tok: 9.5encoded_chars/tok: 2.9
{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 72chars_og/tok: 10.6encoded_chars/tok: 3.5
id; name; role; department
1; Alice; Engineer; Backend
2; Bob; Designer; Frontend
3; Carol; Manager; Product
4; David; Engineer; Infrastructure
5; Eva; Analyst; Data
6; Frank; Engineer; Backend
7; Grace; Designer; Mobile
8; Henry; Manager; Engineering
minemizer (compact)
chars: 224tokens: 77chars_og/tok: 9.9encoded_chars/tok: 2.9
id;name;role;department
1;Alice;Engineer;Backend
2;Bob;Designer;Frontend
3;Carol;Manager;Product
4;David;Engineer;Infrastructure
5;Eva;Analyst;Data
6;Frank;Engineer;Backend
7;Grace;Designer;Mobile
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 741tokens: 252chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id":·2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id":·3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
JSON (min)
chars: 470tokens: 127chars_og/tok: 5.8encoded_chars/tok: 3.7
[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 158chars_og/tok: 4.7encoded_chars/tok: 2.9
- id:·1
· status: active
· user:↵
··· email: alice@example.com
··· name: Alice
- id:·2
· status: inactive
· user:↵
··· email: bob@example.com
··· name: Bob
- id:·3
· status: active
· user:↵
··· email: carol@example.com
··· name: Carol
- id:·4
· status: pending
· user:↵
··· email: david@example.com
··· name: David
- id:·5
· status: active
· user:↵
··· email: eva@example.com
··· name: Eva
... (truncated)
TOON
chars: 527tokens: 166chars_og/tok: 4.5encoded_chars/tok: 3.2
[6]:↵
· - id:·1
··· user:↵
····· name: Alice
····· email: alice@example.com
··· status: active
· - id:·2
··· user:↵
····· name: Bob
····· email: bob@example.com
··· status: inactive
· - id:·3
··· user:↵
····· name: Carol
····· email: carol@example.com
··· status: active
· - id:·4
··· user:↵
····· name: David
····· email: david@example.com
··· status: pending
· - id:·5
··· user:↵
····· name: Eva
····· email: eva@example.com
... (truncated)
TSON
chars: 249tokens: 75chars_og/tok: 9.9encoded_chars/tok: 3.3
{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 77chars_og/tok: 9.6encoded_chars/tok: 3.4
id; user{ name; email}; status
1; { Alice; alice@example.com}; active
2; { Bob; bob@example.com}; inactive
3; { Carol; carol@example.com}; active
4; { David; david@example.com}; pending
5; { Eva; eva@example.com}; active
6; { Frank; frank@example.com}; active
minemizer (compact)
chars: 232tokens: 78chars_og/tok: 9.5encoded_chars/tok: 3.0
id;user{ name;email};status
1;{Alice;alice@example.com};active
2;{Bob;bob@example.com};inactive
3;{Carol;carol@example.com};active
4;{David;david@example.com};pending
5;{Eva;eva@example.com};active
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 610tokens: 217chars_og/tok: 2.8encoded_chars/tok: 2.8
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
JSON (min)
chars: 330tokens: 103chars_og/tok: 5.9encoded_chars/tok: 3.2
[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 149chars_og/tok: 4.1encoded_chars/tok: 2.3
- id:·1
· name: Alice
· skills:↵
· - python
· - go
· - rust
- id:·2
· name: Bob
· skills:↵
· - javascript
· - typescript
- id:·3
· name: Carol
· skills:↵
· - java
· - kotlin
· - scala
· - groovy
- id:·4
· name: David
· skills:↵
· - c
· - cpp
- id:·5
· name: Eva
... (truncated)
TOON
chars: 339tokens: 137chars_og/tok: 4.5encoded_chars/tok: 2.5
[6]:↵
· - id:·1
··· name: Alice
··· skills[3]: python,go,rust
· - id:·2
··· name: Bob
··· skills[2]: javascript,typescript
· - id:·3
··· name: Carol
··· skills[4]: java,kotlin,scala,groovy
· - id:·4
··· name: David
··· skills[2]: c,cpp
· - id:·5
··· name: Eva
··· skills[3]: ruby,elixir,erlang
· - id:·6
··· name: Frank
··· skills[1]: swift
TSON
chars: 168tokens: 65chars_og/tok: 9.4encoded_chars/tok: 2.6
{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 71chars_og/tok: 8.6encoded_chars/tok: 2.7
id; name; skills[]↵
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
minemizer (compact)
chars: 165tokens: 70chars_og/tok: 8.7encoded_chars/tok: 2.4
id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 589tokens: 224chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id":·1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Dave",↵
··· "remote": true
· },↵
· {↵
··· "id":·3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id":·5,↵
··· "name": "Grace",
... (truncated)
JSON (min)
chars: 378tokens: 114chars_og/tok: 5.2encoded_chars/tok: 3.3
[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 143chars_og/tok: 4.1encoded_chars/tok: 2.5
- id:·1
· name: Carol
· role: Manager
- id:·2
· name: Dave
· remote: true
- id:·3
· name: Eve
· role: Designer
· team: UX
- department: Engineering
· id:·4
· name: Frank
- id:·5
· name: Grace
· remote: true
· role: Engineer
· team: Platform
- id:·6
· name: Henry
· role: Analyst
- id:·7
· name: Ivy
- department: Sales
· id:·8
... (truncated)
TOON
chars: 414tokens: 153chars_og/tok: 3.8encoded_chars/tok: 2.7
[8]:↵
· - id:·1
··· name: Carol
··· role: Manager
· - id:·2
··· name: Dave
··· remote: true
· - id:·3
··· name: Eve
··· role: Designer
··· team: UX
· - id:·4
··· name: Frank
··· department: Engineering
· - id:·5
··· name: Grace
··· role: Engineer
··· remote: true
··· team: Platform
· - id:·6
··· name: Henry
··· role: Analyst
· - id:·7
··· name: Ivy
· - id:·8
... (truncated)
TSON
chars: 300tokens: 109chars_og/tok: 5.4encoded_chars/tok: 2.8
[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 77chars_og/tok: 7.6encoded_chars/tok: 3.0
id; name; role
1; Carol; Manager
2; Dave; ; remote:True
3; Eve; Designer; team:UX
4; Frank; ; department:Engineering
5; Grace; Engineer; remote:True; team:Platform
6; Henry; Analyst
7; Ivy;
8; Jack; ; department:Sales; remote:False
minemizer (compact)
chars: 207tokens: 77chars_og/tok: 7.6encoded_chars/tok: 2.7
id;name;role
1;Carol;Manager
2;Dave;;remote:True
3;Eve;Designer;team:UX
4;Frank;;department:Engineering
5;Grace;Engineer;remote:True;team:Platform
6;Henry;Analyst
7;Ivy;↵
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 1,611,780tokens: 611,575chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 361,897chars_og/tok: 4.5encoded_chars/tok: 3.2
[{"id":"_","symbol":"gib","name":"\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000 Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIX Protocol"},{"id":"0x","symbol":"zrx","name":"0x Protocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai: AI Smart Contract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0x Leverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonk by Virtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"Avalanche Bridged ZRX (Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsim by Virtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS (Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000x by Virtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10 figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1 Coin Can Change Your Life"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1 community can change your life"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1 dog can change your life"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1Hive Water"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCH yVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1Move Token"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1 narrative can change your life"},{"id":"1-one","symbol":"one","name":"1 (one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUS DAO"},{"id":"1-squirrel","symbol":"peanut","name":"OG Peanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004 PEPE"},{"id":"2025-token","symbol":"2025","name":"2025 TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077 CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23 Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3 years and forget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24K Gold PEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2G Carbon Coin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"Meow Meow Coin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACE ART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
CSV
chars: 603,635tokens: 228,404chars_og/tok: 7.1encoded_chars/tok: 2.6
id,symbol,name
_,gib,·_
000-capital,000,000 Capital
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100
01-token,01,01
0chain,zcn,Zus
0vix-protocol,vix,0VIX Protocol
0x,zrx,0x Protocol
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract
0x678-landwolf-1933,wolf,Landwolf
0xgasless-2,0xgas,0xGasless
0xgen,xgn,0xGen
0x-leverage,oxl,0x Leverage
0xlsd,0xlsd,0xLSD
0xmonk,monk,0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow
0xsim-by-virtuals,sage,0xsim by Virtuals
0xy,0xy,0xy
-10,loong,
1000bonk,1000bonk,1000BONK
1000btt,1000btt,1000BTT
1000cat,1000cat,1000CAT
1000chems,1000cheems,1000CHEMS
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 224,078chars_og/tok: 7.2encoded_chars/tok: 2.7
id symbol name
_ gib·_
000-capital000000 Capital
01111010011110000110001001110100-token0111101001111000011000100111010001111010011110000110001001110100
01-token0101
0chain zcn Zus
0vix-protocol vix0VIX Protocol
0x zrx0x Protocol
0x0-ai-ai-smart-contract0x00x0.ai: AI Smart Contract
0x678-landwolf-1933 wolf Landwolf
0xgasless-20xgas0xGasless
0xgen xgn0xGen
0x-leverageoxl0x Leverage
0xlsd0xlsd0xLSD
0xmonkmonk0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e zrx.e Avalanche Bridged ZRX (Avalanche) ↵
0xshadow0xs0xShadow
0xsim-by-virtuals sage0xsim by Virtuals
0xy0xy0xy
-10loong
1000bonk1000bonk1000BONK
1000btt1000btt1000BTT
1000cat1000cat1000CAT
1000chems1000cheems1000CHEMS
1000mog1000mog1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 409,204chars_og/tok: 3.9encoded_chars/tok: 2.6
- id: _↵
· name:·_
· symbol: gib
- id:·000-capital
· name:·000 Capital
· symbol: '000'↵
- id:·01111010011110000110001001110100-token
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id:·01-token
· name: '01'↵
· symbol: '01'↵
- id:·0chain
· name: Zus
· symbol: zcn
- id:·0vix-protocol
· name:·0VIX Protocol
· symbol: vix
- id:·0x
· name:·0x Protocol
· symbol: zrx
- id:·0x0-ai-ai-smart-contract
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id:·0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 245,044chars_og/tok: 6.6encoded_chars/tok: 2.5
[19332]{id,symbol,name}:↵
· _,gib,·_
··000-capital,"000",000 Capital
··01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
··01-token,"01","01"↵
··0chain,zcn,Zus
··0vix-protocol,vix,0VIX Protocol
··0x,zrx,0x Protocol
··0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
··0x678-landwolf-1933,wolf,Landwolf
··0xgasless-2,0xgas,0xGasless
··0xgen,xgn,0xGen
··0x-leverage,oxl,0x Leverage
··0xlsd,0xlsd,0xLSD
··0xmonk,monk,0xMonk by Virtuals
··0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
··0xshadow,0xs,0xShadow
··0xsim-by-virtuals,sage,0xsim by Virtuals
··0xy,0xy,0xy
· "-10",loong,
··1000bonk,1000bonk,1000BONK
··1000btt,1000btt,1000BTT
··1000cat,1000cat,1000CAT
··1000chems,1000cheems,1000CHEMS
··1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 232,225chars_og/tok: 6.9encoded_chars/tok: 2.6
{@id,symbol,name#19332|_,gib,"·_"|000-capital,"000","000 Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIX Protocol"|0x,zrx,"0x Protocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0x Leverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonk by Virtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"Avalanche Bridged ZRX (Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsim by Virtuals"|0xy,0xy,0xy|"-10",loong,|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS (Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000x by Virtuals"|100-token,100¥,100¥|100xdarren,100x,100xDarren|10-figs,figs,"10 figs"|"-11",,|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1 Coin Can Change Your Life"|1-community-can-change-your-life,community,"1 community can change your life"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1 dog can change your life"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1Hive Water"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCH yVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1Move Token"|1-narrative-can-change-your-life,narrative,"1 narrative can change your life"|1-one,one,"1 (one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUS DAO"|1-squirrel,peanut,"OG Peanut"|1-token,"1","1"|2004-pepe,bog,"2004 PEPE"|2025-token,"2025","2025 TOKEN"|2077-code,"2077","2077 CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23 Turtles"|2-3-years-and-forget,"23","2-3 years and forget"|24k-gold-pepe,goldpepe,"24K Gold PEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2G Carbon Coin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"Meow Meow Coin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACE ART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47th POTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4 Next Unicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TB Coin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-Way Mirror Money"|"-5",��,��|500m-piece-of-paper,paper,"$500M piece of paper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5th Scape"|"-6"," "," "|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",,Voidify|717ai-by-virtuals,wire,"717ai by Virtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",🔶,🔶|8004-dog,dog8004,"8004 Dog"|8008-token,"8008","8008"|888coin,,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8 Ball"|8-bit-coin,coin,"8-Bit Coin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",,|9-5,9-5,9to5|99-bitcoins,99btc,"99 Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16z AI Dog"|a51-finance,a51,"A51 Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaa cat"|aaai_agent-by-virtuals,aaai,"AAAI_agent by Virtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADex Finance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"Aark Digital"|aarna-afi-802v2,"afi·802v2","aarna afi·802v2"|aarna-atv111,atv111,"aarna atv111"|aarna-atv111-arbitrum,atv111,"aarna atv111 (Arbitrum)"|aarna-atv111-sonic,atv111,"aarna atv111 (Sonic)"|aarna-atv-808,atv808,"aarna atv·808"|aarna-atv-usdc,atvusdc,"aarna atv USDC (Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarna atv USDC (Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"Aave AAVE"|aave-amm-bptbalweth,aammbptbalweth,"Aave AMM BptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"Aave AMM BptWBTCWETH"|aave-amm-dai,aammdai,"Aave AMM DAI"|aave-amm-uniaaveweth,aammuniaaveweth,"Aave AMM UniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"Aave AMM UniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"Aave AMM UniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"Aave AMM UniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"Aave AMM UniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"Aave AMM UniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"Aave AMM·
... (truncated)
minemizer
chars: 622,925tokens: 237,950chars_og/tok: 6.8encoded_chars/tok: 2.6
id; symbol; name
_; gib;·_
000-capital;·000;·000 Capital
01111010011110000110001001110100-token;·01111010011110000110001001110100;·01111010011110000110001001110100
01-token;·01;·01
0chain; zcn; Zus
0vix-protocol; vix;·0VIX Protocol
0x; zrx;·0x Protocol
0x0-ai-ai-smart-contract;·0x0;·0x0.ai: AI Smart Contract
0x678-landwolf-1933; wolf; Landwolf
0xgasless-2;·0xgas;·0xGasless
0xgen; xgn;·0xGen
0x-leverage; oxl;·0x Leverage
0xlsd;·0xlsd;·0xLSD
0xmonk; monk;·0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow;·0xs;·0xShadow
0xsim-by-virtuals; sage;·0xsim by Virtuals
0xy;·0xy;·0xy
-10; loong;
1000bonk;·1000bonk;·1000BONK
1000btt;·1000btt;·1000BTT
1000cat;·1000cat;·1000CAT
1000chems;·1000cheems;·1000CHEMS
1000mog;·1000mog;·1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 237,554chars_og/tok: 6.8encoded_chars/tok: 2.5
id;symbol;name
_;gib;·_
000-capital;000;000 Capital
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100
01-token;01;01
0chain;zcn;Zus
0vix-protocol;vix;0VIX Protocol
0x;zrx;0x Protocol
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract
0x678-landwolf-1933;wolf;Landwolf
0xgasless-2;0xgas;0xGasless
0xgen;xgn;0xGen
0x-leverage;oxl;0x Leverage
0xlsd;0xlsd;0xLSD
0xmonk;monk;0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow
0xsim-by-virtuals;sage;0xsim by Virtuals
0xy;0xy;0xy
-10;loong;
1000bonk;1000bonk;1000BONK
1000btt;1000btt;1000BTT
1000cat;1000cat;1000CAT
1000chems;1000cheems;1000CHEMS
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 1,320tokens: 455chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id":·2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
JSON (min)
chars: 760tokens: 246chars_og/tok: 5.4encoded_chars/tok: 3.1
[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 306chars_og/tok: 4.3encoded_chars/tok: 2.7
- id:·1
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC
····· country: USA
··· name: Grace
· tags:↵
· - admin
· - verified
- id:·2
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London
····· country: UK
··· name: Henry
· tags:↵
· - user
- id:·3
· metadata:↵
··· created: '2024-01-05'
... (truncated)
TOON
chars: 881tokens: 304chars_og/tok: 4.3encoded_chars/tok: 2.9
[5]:↵
· - id:·1
··· profile:↵
····· name: Grace
····· location:↵
······· city: NYC
······· country: USA
··· tags[2]: admin,verified
··· metadata:↵
····· created:·2024-01-15
· - id:·2
··· profile:↵
····· name: Henry
····· location:↵
······· city: London
······· country: UK
··· tags[1]: user
··· metadata:↵
····· created:·2024-02-20
····· updated:·2024-03-10
· - id:·3
··· profile:↵
····· name: Ivy
····· location:↵
······· city: Tokyo
... (truncated)
TSON
chars: 453tokens: 203chars_og/tok: 6.5encoded_chars/tok: 2.2
{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 191chars_og/tok: 6.9encoded_chars/tok: 2.2
id; profile{ name; location{ city; country}}; tags[]; metadata{ created; ...}↵
1; { Grace; { NYC; USA}}; [ admin; verified]; {·2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; {·2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; {·2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; {·2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; {·2023-12-01; updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 191chars_og/tok: 6.9encoded_chars/tok: 1.9
id;profile{ name;location{ city;country}};tags[];metadata{ created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
simple_flat.json — Original size: 763 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 763tokens: 264chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
JSON (min)
chars: 522tokens: 137chars_og/tok: 5.6encoded_chars/tok: 3.8
[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id,name,role,department
1,Alice,Engineer,Backend
2,Bob,Designer,Frontend
3,Carol,Manager,Product
4,David,Engineer,Infrastructure
5,Eva,Analyst,Data
6,Frank,Engineer,Backend
7,Grace,Designer,Mobile
8,Henry,Manager,Engineering
TSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id nameroledepartment
1 AliceEngineerBackend
2 BobDesigner Frontend
3 CarolManagerProduct
4 DavidEngineerInfrastructure
5 EvaAnalyst Data
6 FrankEngineerBackend
7GraceDesigner Mobile
8 HenryManagerEngineering
YAML
chars: 489tokens: 169chars_og/tok: 4.5encoded_chars/tok: 2.9
- department: Backend
· id:·1
· name: Alice
· role: Engineer
- department: Frontend
· id:·2
· name: Bob
· role: Designer
- department: Product
· id:·3
· name: Carol
· role: Manager
- department: Infrastructure
· id:·4
· name: David
· role: Engineer
- department: Data
· id:·5
· name: Eva
· role: Analyst
- department: Backend
· id:·6
· name: Frank
· role: Engineer
- department: Mobile
... (truncated)
TOON
chars: 246tokens: 96chars_og/tok: 7.9encoded_chars/tok: 2.6
[8]{id,name,role,department}:↵
··1,Alice,Engineer,Backend
··2,Bob,Designer,Frontend
··3,Carol,Manager,Product
··4,David,Engineer,Infrastructure
··5,Eva,Analyst,Data
··6,Frank,Engineer,Backend
··7,Grace,Designer,Mobile
··8,Henry,Manager,Engineering
TSON
chars: 229tokens: 80chars_og/tok: 9.5encoded_chars/tok: 2.9
{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 72chars_og/tok: 10.6encoded_chars/tok: 3.5
id; name; role; department
1; Alice; Engineer; Backend
2; Bob; Designer; Frontend
3; Carol; Manager; Product
4; David; Engineer; Infrastructure
5; Eva; Analyst; Data
6; Frank; Engineer; Backend
7; Grace; Designer; Mobile
8; Henry; Manager; Engineering
minemizer (compact)
chars: 224tokens: 77chars_og/tok: 9.9encoded_chars/tok: 2.9
id;name;role;department
1;Alice;Engineer;Backend
2;Bob;Designer;Frontend
3;Carol;Manager;Product
4;David;Engineer;Infrastructure
5;Eva;Analyst;Data
6;Frank;Engineer;Backend
7;Grace;Designer;Mobile
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 741tokens: 252chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id":·2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id":·3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
JSON (min)
chars: 470tokens: 127chars_og/tok: 5.8encoded_chars/tok: 3.7
[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 158chars_og/tok: 4.7encoded_chars/tok: 2.9
- id:·1
· status: active
· user:↵
··· email: alice@example.com
··· name: Alice
- id:·2
· status: inactive
· user:↵
··· email: bob@example.com
··· name: Bob
- id:·3
· status: active
· user:↵
··· email: carol@example.com
··· name: Carol
- id:·4
· status: pending
· user:↵
··· email: david@example.com
··· name: David
- id:·5
· status: active
· user:↵
··· email: eva@example.com
··· name: Eva
... (truncated)
TOON
chars: 527tokens: 166chars_og/tok: 4.5encoded_chars/tok: 3.2
[6]:↵
· - id:·1
··· user:↵
····· name: Alice
····· email: alice@example.com
··· status: active
· - id:·2
··· user:↵
····· name: Bob
····· email: bob@example.com
··· status: inactive
· - id:·3
··· user:↵
····· name: Carol
····· email: carol@example.com
··· status: active
· - id:·4
··· user:↵
····· name: David
····· email: david@example.com
··· status: pending
· - id:·5
··· user:↵
····· name: Eva
····· email: eva@example.com
... (truncated)
TSON
chars: 249tokens: 75chars_og/tok: 9.9encoded_chars/tok: 3.3
{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 77chars_og/tok: 9.6encoded_chars/tok: 3.4
id; user{ name; email}; status
1; { Alice; alice@example.com}; active
2; { Bob; bob@example.com}; inactive
3; { Carol; carol@example.com}; active
4; { David; david@example.com}; pending
5; { Eva; eva@example.com}; active
6; { Frank; frank@example.com}; active
minemizer (compact)
chars: 232tokens: 78chars_og/tok: 9.5encoded_chars/tok: 3.0
id;user{ name;email};status
1;{Alice;alice@example.com};active
2;{Bob;bob@example.com};inactive
3;{Carol;carol@example.com};active
4;{David;david@example.com};pending
5;{Eva;eva@example.com};active
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 610tokens: 217chars_og/tok: 2.8encoded_chars/tok: 2.8
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
JSON (min)
chars: 330tokens: 103chars_og/tok: 5.9encoded_chars/tok: 3.2
[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 149chars_og/tok: 4.1encoded_chars/tok: 2.3
- id:·1
· name: Alice
· skills:↵
· - python
· - go
· - rust
- id:·2
· name: Bob
· skills:↵
· - javascript
· - typescript
- id:·3
· name: Carol
· skills:↵
· - java
· - kotlin
· - scala
· - groovy
- id:·4
· name: David
· skills:↵
· - c
· - cpp
- id:·5
· name: Eva
... (truncated)
TOON
chars: 339tokens: 137chars_og/tok: 4.5encoded_chars/tok: 2.5
[6]:↵
· - id:·1
··· name: Alice
··· skills[3]: python,go,rust
· - id:·2
··· name: Bob
··· skills[2]: javascript,typescript
· - id:·3
··· name: Carol
··· skills[4]: java,kotlin,scala,groovy
· - id:·4
··· name: David
··· skills[2]: c,cpp
· - id:·5
··· name: Eva
··· skills[3]: ruby,elixir,erlang
· - id:·6
··· name: Frank
··· skills[1]: swift
TSON
chars: 168tokens: 65chars_og/tok: 9.4encoded_chars/tok: 2.6
{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 71chars_og/tok: 8.6encoded_chars/tok: 2.7
id; name; skills[]↵
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
minemizer (compact)
chars: 165tokens: 70chars_og/tok: 8.7encoded_chars/tok: 2.4
id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 589tokens: 224chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id":·1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Dave",↵
··· "remote": true
· },↵
· {↵
··· "id":·3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id":·5,↵
··· "name": "Grace",
... (truncated)
JSON (min)
chars: 378tokens: 114chars_og/tok: 5.2encoded_chars/tok: 3.3
[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 143chars_og/tok: 4.1encoded_chars/tok: 2.5
- id:·1
· name: Carol
· role: Manager
- id:·2
· name: Dave
· remote: true
- id:·3
· name: Eve
· role: Designer
· team: UX
- department: Engineering
· id:·4
· name: Frank
- id:·5
· name: Grace
· remote: true
· role: Engineer
· team: Platform
- id:·6
· name: Henry
· role: Analyst
- id:·7
· name: Ivy
- department: Sales
· id:·8
... (truncated)
TOON
chars: 414tokens: 153chars_og/tok: 3.8encoded_chars/tok: 2.7
[8]:↵
· - id:·1
··· name: Carol
··· role: Manager
· - id:·2
··· name: Dave
··· remote: true
· - id:·3
··· name: Eve
··· role: Designer
··· team: UX
· - id:·4
··· name: Frank
··· department: Engineering
· - id:·5
··· name: Grace
··· role: Engineer
··· remote: true
··· team: Platform
· - id:·6
··· name: Henry
··· role: Analyst
· - id:·7
··· name: Ivy
· - id:·8
... (truncated)
TSON
chars: 300tokens: 109chars_og/tok: 5.4encoded_chars/tok: 2.8
[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 77chars_og/tok: 7.6encoded_chars/tok: 3.0
id; name; role
1; Carol; Manager
2; Dave; ; remote:True
3; Eve; Designer; team:UX
4; Frank; ; department:Engineering
5; Grace; Engineer; remote:True; team:Platform
6; Henry; Analyst
7; Ivy;
8; Jack; ; department:Sales; remote:False
minemizer (compact)
chars: 207tokens: 77chars_og/tok: 7.6encoded_chars/tok: 2.7
id;name;role
1;Carol;Manager
2;Dave;;remote:True
3;Eve;Designer;team:UX
4;Frank;;department:Engineering
5;Grace;Engineer;remote:True;team:Platform
6;Henry;Analyst
7;Ivy;↵
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 1,611,780tokens: 609,227chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 359,549chars_og/tok: 4.5encoded_chars/tok: 3.2
[{"id":"_","symbol":"gib","name":"\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000 Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIX Protocol"},{"id":"0x","symbol":"zrx","name":"0x Protocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai: AI Smart Contract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0x Leverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonk by Virtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"Avalanche Bridged ZRX (Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsim by Virtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS (Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000x by Virtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10 figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1 Coin Can Change Your Life"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1 community can change your life"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1 dog can change your life"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1Hive Water"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCH yVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1Move Token"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1 narrative can change your life"},{"id":"1-one","symbol":"one","name":"1 (one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUS DAO"},{"id":"1-squirrel","symbol":"peanut","name":"OG Peanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004 PEPE"},{"id":"2025-token","symbol":"2025","name":"2025 TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077 CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23 Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3 years and forget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24K Gold PEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2G Carbon Coin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"Meow Meow Coin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACE ART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
CSV
chars: 603,635tokens: 227,103chars_og/tok: 7.1encoded_chars/tok: 2.7
id,symbol,name
_,gib,·_
000-capital,000,000 Capital
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100
01-token,01,01
0chain,zcn,Zus
0vix-protocol,vix,0VIX Protocol
0x,zrx,0x Protocol
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract
0x678-landwolf-1933,wolf,Landwolf
0xgasless-2,0xgas,0xGasless
0xgen,xgn,0xGen
0x-leverage,oxl,0x Leverage
0xlsd,0xlsd,0xLSD
0xmonk,monk,0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow
0xsim-by-virtuals,sage,0xsim by Virtuals
0xy,0xy,0xy
-10,loong,
1000bonk,1000bonk,1000BONK
1000btt,1000btt,1000BTT
1000cat,1000cat,1000CAT
1000chems,1000cheems,1000CHEMS
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 222,774chars_og/tok: 7.2encoded_chars/tok: 2.7
id symbol name
_ gib·_
000-capital000000 Capital
01111010011110000110001001110100-token0111101001111000011000100111010001111010011110000110001001110100
01-token0101
0chain zcn Zus
0vix-protocol vix0VIX Protocol
0x zrx0x Protocol
0x0-ai-ai-smart-contract0x00x0.ai: AI Smart Contract
0x678-landwolf-1933 wolf Landwolf
0xgasless-20xgas0xGasless
0xgen xgn0xGen
0x-leverageoxl0x Leverage
0xlsd0xlsd0xLSD
0xmonkmonk0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e zrx.e Avalanche Bridged ZRX (Avalanche) ↵
0xshadow0xs0xShadow
0xsim-by-virtuals sage0xsim by Virtuals
0xy0xy0xy
-10loong
1000bonk1000bonk1000BONK
1000btt1000btt1000BTT
1000cat1000cat1000CAT
1000chems1000cheems1000CHEMS
1000mog1000mog1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 407,827chars_og/tok: 4.0encoded_chars/tok: 2.6
- id: _↵
· name:·_
· symbol: gib
- id:·000-capital
· name:·000 Capital
· symbol: '000'↵
- id:·01111010011110000110001001110100-token
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id:·01-token
· name: '01'↵
· symbol: '01'↵
- id:·0chain
· name: Zus
· symbol: zcn
- id:·0vix-protocol
· name:·0VIX Protocol
· symbol: vix
- id:·0x
· name:·0x Protocol
· symbol: zrx
- id:·0x0-ai-ai-smart-contract
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id:·0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 243,750chars_og/tok: 6.6encoded_chars/tok: 2.6
[19332]{id,symbol,name}:↵
· _,gib,·_
··000-capital,"000",000 Capital
··01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
··01-token,"01","01"↵
··0chain,zcn,Zus
··0vix-protocol,vix,0VIX Protocol
··0x,zrx,0x Protocol
··0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
··0x678-landwolf-1933,wolf,Landwolf
··0xgasless-2,0xgas,0xGasless
··0xgen,xgn,0xGen
··0x-leverage,oxl,0x Leverage
··0xlsd,0xlsd,0xLSD
··0xmonk,monk,0xMonk by Virtuals
··0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
··0xshadow,0xs,0xShadow
··0xsim-by-virtuals,sage,0xsim by Virtuals
··0xy,0xy,0xy
· "-10",loong,
··1000bonk,1000bonk,1000BONK
··1000btt,1000btt,1000BTT
··1000cat,1000cat,1000CAT
··1000chems,1000cheems,1000CHEMS
··1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 230,921chars_og/tok: 7.0encoded_chars/tok: 2.6
{@id,symbol,name#19332|_,gib,"·_"|000-capital,"000","000 Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIX Protocol"|0x,zrx,"0x Protocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0x Leverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonk by Virtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"Avalanche Bridged ZRX (Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsim by Virtuals"|0xy,0xy,0xy|"-10",loong,|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS (Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000x by Virtuals"|100-token,100¥,100¥|100xdarren,100x,100xDarren|10-figs,figs,"10 figs"|"-11",,|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1 Coin Can Change Your Life"|1-community-can-change-your-life,community,"1 community can change your life"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1 dog can change your life"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1Hive Water"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCH yVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1Move Token"|1-narrative-can-change-your-life,narrative,"1 narrative can change your life"|1-one,one,"1 (one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUS DAO"|1-squirrel,peanut,"OG Peanut"|1-token,"1","1"|2004-pepe,bog,"2004 PEPE"|2025-token,"2025","2025 TOKEN"|2077-code,"2077","2077 CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23 Turtles"|2-3-years-and-forget,"23","2-3 years and forget"|24k-gold-pepe,goldpepe,"24K Gold PEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2G Carbon Coin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"Meow Meow Coin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACE ART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47th POTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4 Next Unicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TB Coin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-Way Mirror Money"|"-5",��,��|500m-piece-of-paper,paper,"$500M piece of paper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5th Scape"|"-6"," "," "|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",,Voidify|717ai-by-virtuals,wire,"717ai by Virtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",,|8004-dog,dog8004,"8004 Dog"|8008-token,"8008","8008"|888coin,,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8 Ball"|8-bit-coin,coin,"8-Bit Coin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",,|9-5,9-5,9to5|99-bitcoins,99btc,"99 Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16z AI Dog"|a51-finance,a51,"A51 Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaa cat"|aaai_agent-by-virtuals,aaai,"AAAI_agent by Virtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADex Finance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"Aark Digital"|aarna-afi-802v2,"afi·802v2","aarna afi·802v2"|aarna-atv111,atv111,"aarna atv111"|aarna-atv111-arbitrum,atv111,"aarna atv111 (Arbitrum)"|aarna-atv111-sonic,atv111,"aarna atv111 (Sonic)"|aarna-atv-808,atv808,"aarna atv·808"|aarna-atv-usdc,atvusdc,"aarna atv USDC (Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarna atv USDC (Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"Aave AAVE"|aave-amm-bptbalweth,aammbptbalweth,"Aave AMM BptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"Aave AMM BptWBTCWETH"|aave-amm-dai,aammdai,"Aave AMM DAI"|aave-amm-uniaaveweth,aammuniaaveweth,"Aave AMM UniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"Aave AMM UniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"Aave AMM UniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"Aave AMM UniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"Aave AMM UniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"Aave AMM UniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"Aave AMM·
... (truncated)
minemizer
chars: 622,925tokens: 236,563chars_og/tok: 6.8encoded_chars/tok: 2.6
id; symbol; name
_; gib;·_
000-capital;·000;·000 Capital
01111010011110000110001001110100-token;·01111010011110000110001001110100;·01111010011110000110001001110100
01-token;·01;·01
0chain; zcn; Zus
0vix-protocol; vix;·0VIX Protocol
0x; zrx;·0x Protocol
0x0-ai-ai-smart-contract;·0x0;·0x0.ai: AI Smart Contract
0x678-landwolf-1933; wolf; Landwolf
0xgasless-2;·0xgas;·0xGasless
0xgen; xgn;·0xGen
0x-leverage; oxl;·0x Leverage
0xlsd;·0xlsd;·0xLSD
0xmonk; monk;·0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow;·0xs;·0xShadow
0xsim-by-virtuals; sage;·0xsim by Virtuals
0xy;·0xy;·0xy
-10; loong;
1000bonk;·1000bonk;·1000BONK
1000btt;·1000btt;·1000BTT
1000cat;·1000cat;·1000CAT
1000chems;·1000cheems;·1000CHEMS
1000mog;·1000mog;·1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 236,253chars_og/tok: 6.8encoded_chars/tok: 2.5
id;symbol;name
_;gib;·_
000-capital;000;000 Capital
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100
01-token;01;01
0chain;zcn;Zus
0vix-protocol;vix;0VIX Protocol
0x;zrx;0x Protocol
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract
0x678-landwolf-1933;wolf;Landwolf
0xgasless-2;0xgas;0xGasless
0xgen;xgn;0xGen
0x-leverage;oxl;0x Leverage
0xlsd;0xlsd;0xLSD
0xmonk;monk;0xMonk by Virtuals
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow
0xsim-by-virtuals;sage;0xsim by Virtuals
0xy;0xy;0xy
-10;loong;
1000bonk;1000bonk;1000BONK
1000btt;1000btt;1000BTT
1000cat;1000cat;1000CAT
1000chems;1000cheems;1000CHEMS
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 1,320tokens: 427chars_og/tok: 3.1encoded_chars/tok: 3.1
[↵
· {↵
··· "id":·1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id":·2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
JSON (min)
chars: 760tokens: 218chars_og/tok: 6.1encoded_chars/tok: 3.5
[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 278chars_og/tok: 4.7encoded_chars/tok: 2.9
- id:·1
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC
····· country: USA
··· name: Grace
· tags:↵
· - admin
· - verified
- id:·2
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London
····· country: UK
··· name: Henry
· tags:↵
· - user
- id:·3
· metadata:↵
··· created: '2024-01-05'
... (truncated)
TOON
chars: 881tokens: 276chars_og/tok: 4.8encoded_chars/tok: 3.2
[5]:↵
· - id:·1
··· profile:↵
····· name: Grace
····· location:↵
······· city: NYC
······· country: USA
··· tags[2]: admin,verified
··· metadata:↵
····· created:·2024-01-15
· - id:·2
··· profile:↵
····· name: Henry
····· location:↵
······· city: London
······· country: UK
··· tags[1]: user
··· metadata:↵
····· created:·2024-02-20
····· updated:·2024-03-10
· - id:·3
··· profile:↵
····· name: Ivy
····· location:↵
······· city: Tokyo
... (truncated)
TSON
chars: 453tokens: 175chars_og/tok: 7.5encoded_chars/tok: 2.6
{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 163chars_og/tok: 8.1encoded_chars/tok: 2.6
id; profile{ name; location{ city; country}}; tags[]; metadata{ created; ...}↵
1; { Grace; { NYC; USA}}; [ admin; verified]; {·2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; {·2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; {·2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; {·2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; {·2023-12-01; updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 163chars_og/tok: 8.1encoded_chars/tok: 2.2
id;profile{ name;location{ city;country}};tags[];metadata{ created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}